import scrapy
import copy
from lxml import etree
from imgtest.items import ImgtestItem


class HuabanSpider(scrapy.Spider):
    name = 'huaban'
    allowed_domains = ['xiuaa.com']
    start_urls = ['https://www.xiuaa.com/']
    base_url = 'https://www.xiuaa.com'

    def parse(self, response):
        """
        提取标签名称和链接
        :param response:
        :return:
        """
        tags = response.xpath("//div[@id='SubBox']/a")
        for i in tags:
            item = ImgtestItem()
            tag_href = i.xpath(".//@href").extract()[0]
            tag_name = i.xpath(".//@title").extract()[0]
            item['tag_name'] = tag_name
            item['tag_href'] = tag_href
            yield scrapy.Request(url=self.base_url + item['tag_href'], meta={'item': copy.deepcopy(item)}, callback=self.parse_page)

    def parse_page(self, response):
        """
        提取标签下链接
        :param response:
        :return:
        """
        item = response.meta['item']
        tags = response.xpath("//div[@class='piax']/h2/a").extract()
        if tags:
            for tag in tags:
                html = etree.HTML(tag)
                item['album_href'] = html.xpath("//a/@href")[0]
                item['album_name'] = html.xpath("//a/@title")[0].strip()
                yield scrapy.Request(url=self.base_url+item['album_href'],meta={'item': copy.deepcopy(item)},callback=self.parse_my_img)

        next_links = response.xpath("//a[text()='下一页']/@href").extract()
        if next_links:
            yield scrapy.Request(url=self.base_url+next_links[0],meta={'item': copy.deepcopy(item)},callback=self.parse_page)


    def parse_album(self, response):
        """
        提取专辑名称和专辑链接
        :param response:
        :return:
        """

        albums = response.xpath(".//*[@class='pic']")
        for album in albums:
            album_href = album.xpath(".//a/@href").extract()[0]
            album_name = album.xpath(".//a/img/@alt").extract()[0]
            item = response.meta['item']
            item['album_name'] = album_name
            item['album_href'] = album_href

            yield scrapy.Request(url=item['album_href'], meta={'item': copy.deepcopy(item)}, callback=self.parse_img)

    def parse_img(self, response):
        """
        提取图片名称和链接
        :param response:
        :return:
        """

        img_list = response.xpath(".//p/img")

        for img in img_list:
            item = response.meta['item']
            img_title = img.xpath(".//@alt").extract()[0]
            if img_title == '':
                for i in range(1, len(img_list) + 1):
                    img_title = item['album_name'] + '_' + str(i)
            else:
                img_title = img_title
            img_urls = img.xpath(".//@src").extract()
            img_src = img.xpath(".//@src").extract()[0]
            item['img_title'] = img_title
            item['img_src'] = img_src
            item['img_urls'] = img_urls

            yield copy.deepcopy(item)

    def parse_my_img(self,response):
        item = response.meta['item']
        imgs = response.xpath("//div[@id='bigpic']//img").extract()
        if imgs:
            html = etree.HTML(imgs[0])
            item['img_src'] = html.xpath("//img/@src")[0]
            item['img_title'] = html.xpath("//img/@alt")[0]
            item['img_urls'] = html.xpath("//img/@src")
            yield copy.deepcopy(item)

        next_links = response.xpath("//a[text()='下一页']/@href").extract()
        if next_links:
            yield scrapy.Request(url=self.base_url + next_links[0], meta={'item': copy.deepcopy(item)},
                                 callback=self.parse_my_img)
